package com.bigdata.collector.labels;

import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.bigdata.collector.model.Page;
import com.bigdata.collector.util.HttpUtils;

/**
 * 
 * @author 岳海亮
 * @email yhl@feheadline.com
 * @date 2015年3月18日
 */
public class Collector4Baidu extends AbstractCollector {
	private static Logger logger = Logger.getLogger(Collector4Baidu.class);

	public Collector4Baidu(String seed) {
		super(seed);
	}

	public static void main(String[] args) {
		String seed = "http://baike.baidu.com/fenlei/%E7%BB%8F%E6%B5%8E";
		Collector4Baidu ct = new Collector4Baidu(seed);
		ct.run();
	}

	@Override
	public void extractURLs(Document doc) {
		try {
			Elements es = doc.getElementsByClass("category-title");
			for (Element ae : es.select("a")) {
				String url = "http://baike.baidu.com" + ae.attr("href").trim();
				if (!crawled.contains(url) && !candidate.contains(url)) {
					candidate.add(url);
					continue;
				}
				String tag = ae.text().trim();
				save2DB(tag, "0");
			}
			es = doc.getElementsByClass("nslog:7452");
			for (Element ae : es) {
				String url = ae.attr("href").trim();
				if (!crawled.contains(url) && !candidate.contains(url))
					candidate.add("http://baike.baidu.com" + url);
			}
		} catch (Exception e) {
			logger.error(ExceptionUtils.getStackTrace(e));
		}
	}

	@Override
	public void crawler(String url) {
		try {
			int pageNum = 1;
			String previous = "";
			while (true) {
				String path = url + "?index=" + pageNum;
				Page page = HttpUtils.fetchHttpResponse(path);
				logger.info("page=" + pageNum);
				String content = new String(page.content, "utf-8");
				if (previous.equals(content))
					break;
				Document doc = Jsoup.parse(content);
				Elements es = doc.select("h3[class=title]");
				String drill = es.first().siblingElements().first().text();
				logger.info(drill);
				if (drill.indexOf("经济") < 0)
					break;
				if (!extractTags(doc))
					break;
				if (pageNum == 1)
					extractURLs(doc);
				pageNum++;
				previous = content;
			}
		} catch (Exception e) {
			logger.error(ExceptionUtils.getStackTrace(e));
		}

	}

	@Override
	public boolean extractTags(Document doc) {
		Elements es = doc.select("div ul li a[class=title nslog:7450]");
		if (es.size() == 0)
			return false;
		for (Element e : es) {
			save2DB(e.text().toString(), "0");
		}
		es = doc.getElementsByClass("nslog:7452");
		if (es.size() > 0)
			return false;
		return true;
	}
}
